{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Debugging Models in TensorFlow <a class=\"tocSkip\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow:1.4.1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(123)\n",
    "print(\"TensorFlow:{}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASETSLIB_HOME = '../datasetslib'\n",
    "import sys\n",
    "if not DATASETSLIB_HOME in sys.path:\n",
    "    sys.path.append(DATASETSLIB_HOME)\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import datasetslib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasetslib.datasets_root = os.path.join(os.path.expanduser('~'),'datasets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /home/armando/datasets/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(os.path.join(datasetslib.datasets_root,'mnist'), one_hot=True)\n",
    "\n",
    "x_train = mnist.train.images\n",
    "x_test = mnist.test.images\n",
    "y_train = mnist.train.labels\n",
    "y_test = mnist.test.labels\n",
    "\n",
    "# parameters\n",
    "n_y = 10  # 0-9 digits\n",
    "n_x = 784  # total pixels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tf.Print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mlp(x, num_inputs, num_outputs,num_layers,num_neurons):\n",
    "    w=[]\n",
    "    b=[]\n",
    "    for i in range(num_layers):\n",
    "        # weights\n",
    "        w.append(tf.Variable(tf.random_normal( \n",
    "                              [num_inputs if i==0 else num_neurons[i-1], \n",
    "                               num_neurons[i]]), \n",
    "                             name=\"w_{0:04d}\".format(i) \n",
    "                            ) \n",
    "                ) \n",
    "        # biases\n",
    "        b.append(tf.Variable(tf.random_normal( \n",
    "                              [num_neurons[i]]), \n",
    "                             name=\"b_{0:04d}\".format(i) \n",
    "                            ) \n",
    "                )                   \n",
    "    w.append(tf.Variable(tf.random_normal(\n",
    "                          [num_neurons[num_layers-1] if num_layers > 0 else num_inputs,\n",
    "                           num_outputs]),name=\"w_out\"))\n",
    "    b.append(tf.Variable(tf.random_normal([num_outputs]),name=\"b_out\"))\n",
    "    \n",
    "    # x is input layer\n",
    "    layer = x\n",
    "    # add hidden layers\n",
    "    for i in range(num_layers):\n",
    "        layer = tf.nn.relu(tf.matmul(layer, w[i]) + b[i])\n",
    "    # add output layer\n",
    "    layer = tf.matmul(layer, w[num_layers]) + b[num_layers]\n",
    "    \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0000   loss = 6.716061\n",
      "epoch: 0001   loss = 2.002921\n",
      "epoch: 0002   loss = 1.733159\n",
      "epoch: 0003   loss = 1.596843\n",
      "epoch: 0004   loss = 1.504387\n",
      "epoch: 0005   loss = 1.435767\n",
      "epoch: 0006   loss = 1.378511\n",
      "epoch: 0007   loss = 1.329088\n",
      "epoch: 0008   loss = 1.287037\n",
      "epoch: 0009   loss = 1.249800\n"
     ]
    }
   ],
   "source": [
    "num_layers = 2\n",
    "num_neurons = [16,32]\n",
    "learning_rate = 0.01\n",
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(mnist.train.num_examples/batch_size)\n",
    "\n",
    "# input images\n",
    "x_p = tf.placeholder(dtype=tf.float32, name=\"x_p\", shape=[None, n_x]) \n",
    "# target output\n",
    "y_p = tf.placeholder(dtype=tf.float32, name=\"y_p\", shape=[None, n_y]) \n",
    "\n",
    "model = mlp(x=x_p, \n",
    "            num_inputs=n_x, \n",
    "            num_outputs=n_y, \n",
    "            num_layers=num_layers, \n",
    "            num_neurons=num_neurons)\n",
    "\n",
    "model = tf.Print(input_=model,\n",
    "                 data=[tf.argmax(model,1)],\n",
    "                 message='y_hat=',\n",
    "                 summarize=10,\n",
    "                 first_n=5\n",
    "                )\n",
    "\n",
    "# loss function\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=y_p))\n",
    "# optimizer function\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "optimizer = optimizer.minimize(loss)\n",
    "\n",
    "#predictions_check = tf.equal(tf.argmax(model,1), tf.argmax(y,1))\n",
    "#accuracy_function = tf.reduce_mean(tf.cast(predictions_check, tf.float32))\n",
    "\n",
    "with tf.Session() as tfs:\n",
    "        tfs.run(tf.global_variables_initializer())\n",
    "        for epoch in range(n_epochs):\n",
    "            epoch_loss = 0.0\n",
    "            for batch in range(n_batches):\n",
    "                X_batch, Y_batch = mnist.train.next_batch(batch_size)\n",
    "                feed_dict={x_p: X_batch, y_p: Y_batch}\n",
    "                _,batch_loss = tfs.run([optimizer,loss], \n",
    "                                       feed_dict = feed_dict\n",
    "                                      )\n",
    "                epoch_loss += batch_loss \n",
    "            average_loss = epoch_loss / n_batches\n",
    "            print(\"epoch: {0:04d}   loss = {1:0.6f}\".format(epoch,average_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tf.Assert()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mlp(x, num_inputs, num_outputs,num_layers,num_neurons):\n",
    "    w=[]\n",
    "    b=[]\n",
    "    for i in range(num_layers):\n",
    "        # weights\n",
    "        w.append(tf.Variable(tf.random_normal( \n",
    "                              [num_inputs if i==0 else num_neurons[i-1], \n",
    "                               num_neurons[i]]), \n",
    "                             name=\"w_{0:04d}\".format(i) \n",
    "                            ) \n",
    "                ) \n",
    "        # biases\n",
    "        b.append(tf.Variable(tf.random_normal( \n",
    "                              [num_neurons[i]]), \n",
    "                             name=\"b_{0:04d}\".format(i) \n",
    "                            ) \n",
    "                )                   \n",
    "    w.append(tf.Variable(tf.random_normal(\n",
    "                          [num_neurons[num_layers-1] if num_layers > 0 else num_inputs,\n",
    "                           num_outputs]),name=\"w_out\"))\n",
    "    b.append(tf.Variable(tf.random_normal([num_outputs]),name=\"b_out\"))\n",
    "    \n",
    "    assert_op = tf.Assert(tf.reduce_all(tf.greater_equal(x,0)),[x])\n",
    "    with tf.control_dependencies([assert_op]):\n",
    "        # x is input layer\n",
    "        layer = x\n",
    "        # add hidden layers\n",
    "        for i in range(num_layers):\n",
    "            layer = tf.nn.relu(tf.matmul(layer, w[i]) + b[i])\n",
    "        # add output layer\n",
    "        layer = tf.matmul(layer, w[num_layers]) + b[num_layers]\n",
    "    \n",
    "    return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0000   loss = 7.803496\n",
      "epoch: 0001   loss = 2.405806\n",
      "epoch: 0002   loss = 2.057570\n",
      "epoch: 0003   loss = 1.884273\n",
      "epoch: 0004   loss = 1.768692\n",
      "epoch: 0005   loss = 1.672444\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "assertion failed: [[-2 0 0]...]\n\t [[Node: Assert/AssertGuard/Assert = Assert[T=[DT_FLOAT], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](Assert/AssertGuard/Assert/Switch/_7, Assert/AssertGuard/Assert/Switch_1)]]\n\t [[Node: Assert/AssertGuard/Assert/_10 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device_incarnation=1, tensor_name=\"edge_18_Assert/AssertGuard/Assert\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"]()]]\n\nCaused by op 'Assert/AssertGuard/Assert', defined at:\n  File \"/usr/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.5/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-10-265ff61fe705>\", line 17, in <module>\n    num_neurons=num_neurons)\n  File \"<ipython-input-8-437a0fa9085d>\", line 23, in mlp\n    assert_op = tf.Assert(tf.reduce_all(tf.greater_equal(x,0)),[x])\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/tf_should_use.py\", line 107, in wrapped\n    return _add_should_use_warning(fn(*args, **kwargs))\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 134, in Assert\n    condition, no_op, true_assert, name=\"AssertGuard\")\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py\", line 316, in new_func\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 1864, in cond\n    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 1725, in BuildCondBranch\n    original_result = fn()\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 132, in true_assert\n    condition, data, summarize, name=\"Assert\")\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_logging_ops.py\", line 47, in _assert\n    name=name)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py\", line 2956, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py\", line 1470, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInvalidArgumentError (see above for traceback): assertion failed: [[-2 0 0]...]\n\t [[Node: Assert/AssertGuard/Assert = Assert[T=[DT_FLOAT], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](Assert/AssertGuard/Assert/Switch/_7, Assert/AssertGuard/Assert/Switch_1)]]\n\t [[Node: Assert/AssertGuard/Assert/_10 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device_incarnation=1, tensor_name=\"edge_18_Assert/AssertGuard/Assert\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"]()]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[1;32m    472\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[1;32m    474\u001b[0m     \u001b[0;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: assertion failed: [[-2 0 0]...]\n\t [[Node: Assert/AssertGuard/Assert = Assert[T=[DT_FLOAT], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](Assert/AssertGuard/Assert/Switch/_7, Assert/AssertGuard/Assert/Switch_1)]]\n\t [[Node: Assert/AssertGuard/Assert/_10 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device_incarnation=1, tensor_name=\"edge_18_Assert/AssertGuard/Assert\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"]()]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-265ff61fe705>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     44\u001b[0m                 \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx_p\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_p\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mY_batch\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m                 _,batch_loss = tfs.run([optimizer,loss], \n\u001b[0;32m---> 46\u001b[0;31m                                        \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m                                       )\n\u001b[1;32m     48\u001b[0m                 \u001b[0mepoch_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mbatch_loss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1334\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1335\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1336\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1338\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: assertion failed: [[-2 0 0]...]\n\t [[Node: Assert/AssertGuard/Assert = Assert[T=[DT_FLOAT], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](Assert/AssertGuard/Assert/Switch/_7, Assert/AssertGuard/Assert/Switch_1)]]\n\t [[Node: Assert/AssertGuard/Assert/_10 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device_incarnation=1, tensor_name=\"edge_18_Assert/AssertGuard/Assert\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"]()]]\n\nCaused by op 'Assert/AssertGuard/Assert', defined at:\n  File \"/usr/lib/python3.5/runpy.py\", line 184, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/usr/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/usr/local/lib/python3.5/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/usr/local/lib/python3.5/dist-packages/ipykernel/zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/usr/local/lib/python3.5/dist-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-10-265ff61fe705>\", line 17, in <module>\n    num_neurons=num_neurons)\n  File \"<ipython-input-8-437a0fa9085d>\", line 23, in mlp\n    assert_op = tf.Assert(tf.reduce_all(tf.greater_equal(x,0)),[x])\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/tf_should_use.py\", line 107, in wrapped\n    return _add_should_use_warning(fn(*args, **kwargs))\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 134, in Assert\n    condition, no_op, true_assert, name=\"AssertGuard\")\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py\", line 316, in new_func\n    return func(*args, **kwargs)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 1864, in cond\n    orig_res_f, res_f = context_f.BuildCondBranch(false_fn)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 1725, in BuildCondBranch\n    original_result = fn()\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/control_flow_ops.py\", line 132, in true_assert\n    condition, data, summarize, name=\"Assert\")\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_logging_ops.py\", line 47, in _assert\n    name=name)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py\", line 2956, in create_op\n    op_def=op_def)\n  File \"/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py\", line 1470, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInvalidArgumentError (see above for traceback): assertion failed: [[-2 0 0]...]\n\t [[Node: Assert/AssertGuard/Assert = Assert[T=[DT_FLOAT], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](Assert/AssertGuard/Assert/Switch/_7, Assert/AssertGuard/Assert/Switch_1)]]\n\t [[Node: Assert/AssertGuard/Assert/_10 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device_incarnation=1, tensor_name=\"edge_18_Assert/AssertGuard/Assert\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"]()]]\n"
     ]
    }
   ],
   "source": [
    "num_layers = 2\n",
    "num_neurons = [16,32]\n",
    "learning_rate = 0.01\n",
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(mnist.train.num_examples/batch_size)\n",
    "\n",
    "# input images\n",
    "x_p = tf.placeholder(dtype=tf.float32, name=\"x_p\", shape=[None, n_x]) \n",
    "# target output\n",
    "y_p = tf.placeholder(dtype=tf.float32, name=\"y_p\", shape=[None, n_y]) \n",
    "\n",
    "model = mlp(x=x_p, \n",
    "            num_inputs=n_x, \n",
    "            num_outputs=n_y, \n",
    "            num_layers=num_layers, \n",
    "            num_neurons=num_neurons)\n",
    "\n",
    "model = tf.Print(input_=model,\n",
    "                 data=[tf.argmax(model,1)],\n",
    "                 message='y_hat=',\n",
    "                 summarize=10,\n",
    "                 first_n=5\n",
    "                )\n",
    "\n",
    "# loss function\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model, labels=y_p))\n",
    "# optimizer function\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "optimizer = optimizer.minimize(loss)\n",
    "\n",
    "#predictions_check = tf.equal(tf.argmax(model,1), tf.argmax(y,1))\n",
    "#accuracy_function = tf.reduce_mean(tf.cast(predictions_check, tf.float32))\n",
    "\n",
    "with tf.Session() as tfs:\n",
    "        tfs.run(tf.global_variables_initializer())\n",
    "        for epoch in range(n_epochs):\n",
    "            epoch_loss = 0.0\n",
    "            for batch in range(n_batches):\n",
    "                X_batch, Y_batch = mnist.train.next_batch(batch_size)\n",
    "                if epoch > 5:\n",
    "                    X_batch = np.copy(X_batch)\n",
    "                    X_batch[0,0]=-2\n",
    "                feed_dict={x_p: X_batch, y_p: Y_batch}\n",
    "                _,batch_loss = tfs.run([optimizer,loss], \n",
    "                                       feed_dict = feed_dict\n",
    "                                      )\n",
    "                epoch_loss += batch_loss \n",
    "            average_loss = epoch_loss / n_batches\n",
    "            print(\"epoch: {0:04d}   loss = {1:0.6f}\".format(epoch,average_loss))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
